package DBOpt;

import java.util.*;
import java.util.Date;
import java.io.*;

import org.w3c.dom.*;
import org.xml.sax.SAXException;

import javax.xml.parsers.*;
import java.sql.*;
import java.util.List;

import timeline.WgetRunnable;
import twitter4j.Status;
import twitter4j.Twitter;
import java.text.SimpleDateFormat;


//import com.mysql.jdbc.*;

public class GetMsgIDApp {

 //
 Connection con;
 DocumentBuilderFactory factory;
 
  SimpleDateFormat createAtDateFormat;
  SimpleDateFormat outputTimestampFormat;
  SimpleDateFormat outputDateFormat,outputTimeFormat;
  
  protected  GetMsgIDApp(){
	//convert datetime
	// Format: Mon Sep 01 04:00:53 +0000 2008
	 createAtDateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy",Locale.US);
	 outputTimestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss",Locale.US);
	 outputDateFormat = new SimpleDateFormat("yyyy-MM-dd",Locale.US);
	 outputTimeFormat = new SimpleDateFormat("HH:mm:ss",Locale.US);
	 
 }
	
 /* public boolean Connect(String host, String user, String password)
  {
	  factory = DocumentBuilderFactory.newInstance();  
	  try {
	      Class.forName("com.mysql.jdbc.Driver").newInstance();
	      
	  }
	  
	    catch( Exception e ) {
	      System.out.println("Failed to load mySQL driver.");
	      e.printStackTrace();
	      return false;
	    }
	    try {
	    	
	    String url = "jdbc:mysql://"+host.trim()+"/twitter";
	     
	      con = DriverManager.getConnection(url, user, password);
	      //con = DriverManager.getConnection(url);
	      System.out.println("Connect Succ!");
	      return true;
	    }
	    catch( Exception e ) {
	      e.printStackTrace();
	      return false;
	    }
	  }
  */
  public void insertAllFile(String rootDirName)
  {
	  
      File rootDir = new File(rootDirName);
      Calendar cal = Calendar.getInstance();
      FileWriter  msgListFile,finishedDirFile;
      factory = DocumentBuilderFactory.newInstance();
      if (!rootDir.isDirectory()) 
      {
          // Either dir does not exist or is not a directory
    	  System.out.println(rootDirName+"is not a valid directory!");
    	  return;
      } 
    
      
      else 
      {
    	  String[] dir_children = rootDir.list();
    	  Arrays.sort(dir_children);

          // Use the factory to create a builder
    	  
    	  DocumentBuilder builder;
    	  try
    	  {
             builder = factory.newDocumentBuilder();
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
          
    	    	  
    	  try
    	  {
    	   
    	   msgListFile = new FileWriter ("./duplicateFile.txt", true);
    	   System.out.println("creat msgIDlist");
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
    	  ////
    	  int index = 0;
    	  try
    	  {
    		  FileInputStream in = new FileInputStream("./.getmsgfinished_list");
        	  BufferedReader br = new BufferedReader(new InputStreamReader(in));
        	  		
        	  String strLine = null, tmp;

        	  while ((tmp = br.readLine()) != null)
        	  {
        	     strLine = tmp;
        	  }
        	  String lastLine = strLine;
        	  System.out.println("Previous Time:"+lastLine);
        	  if (lastLine != null){
        		  for(;index<dir_children.length;index++)
            	  {
            		String date = dir_children[index];
            		if (date.compareTo(lastLine)>0) break;
            		
            	  }
            	  if (index == dir_children.length)
            	  {
            		System.out.println ("No new data in incoming dir!");
            		return;
            	  }  
          		
        	  }
	 
    	  }
    	  catch (FileNotFoundException e)
    	  {
    		  System.out.println ("No previous finished dir");
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
    	  
    	  try
    	  {
    	   
    	   finishedDirFile = new FileWriter ("./.getmsgfinished_list", true);
    	   System.out.println("creart finishfile");
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
    	  
    	  System.out.println ("*** Start processing from:"+dir_children[index]+" ****");

    	  ////
    	  
    	  for (; index<dir_children.length; index++) 
          {
              String subDirName = rootDirName+"/"+dir_children[index];
              File subDir = new File(subDirName);
              if (!subDir.isDirectory())
              {
            	  System.out.println(subDir+" is not a sub directory! ignore it!");  
              }
              else
              {
            	  String[] file_children = subDir.list();
            	  Arrays.sort(file_children);
            	  
            	  Set<Integer> currentIDList;
            	  Set<Integer> previousIDList = null;
            	  
            	  for (int j=0; j<file_children.length; j++) 
                  {
            		  
           		      String s= file_children[j];
            		  int surfix_index =  s.lastIndexOf("xml");
            		  if (surfix_index != file_children[j].length()-3 ) 
            	      {
            	    	  System.out.println("Non-XML File:"+file_children[j]);
            	    	
            	      }
            	      else
            	      {
            	    	
            	    	  currentIDList = new HashSet <Integer>();
            	    	  currentIDList.clear();
            	    	  String filename = subDirName+"/"+file_children[j];
                          File file = new File(filename);
                          System.out.println("XML File:"+filename);
                          
                          
                          //Document doc = builder.parse(new java.io.ByteArrayInputStream(file.trim().getBytes("UTF8")));
                          List<Status>  statusList = null;
                          try{
                          Document doc = builder.parse(file);
                          // Get a list of all elements in the document
                          Twitter twitter = new Twitter();
                          statusList = Status.constructStatuses(doc, twitter);
                          }
                          catch( SAXException se ) 
                 	      {
                 	        System.out.println("XML Parse error:"+se.getMessage());
                 	        try{
                 	          msgListFile.write(filename+"\t"+"XML Parse error: "+se.getMessage()+"\n");
                 	          msgListFile.flush();
                 	         }
                 	        catch( Exception e1 ) 
                  	         {
                  	         e1.printStackTrace();
                  	          }
                 	        
                 	        continue;
                 	      }
                           catch( Exception e ) 
                 	      {
                        	   try{
                      	          msgListFile.write(filename+"\t"+"UNKNOW Error: "+e.getMessage()+"\n");
                      	          msgListFile.flush();
                      	         }
                      	        catch( Exception e1 ) 
                       	         {
                       	         e1.printStackTrace();
                       	          }
                 	           e.printStackTrace();
                 	        continue;
                 	       }
                          
                          for (Status status : statusList) {
                        	// String SQL="";
                        	 //String userName, userScreenName, userLocation,userDescription, userProfileImageURL,userURL,text;
                        	 //String source,createDate="",createTime="";
                            //int id=0,isTruncated ,InReplyToStatusId, getInReplyToUserId,isFavorited,userId=0;
                        	  int id=0;
                        	try{
                        	
                          	String CreatedAt = status.getCreatedAt();
                          	id = status.getId();
                          	currentIDList.add(id);
                          	                          
                            }
                            catch( Exception e ) 
                 	        {
                        	  try{
                      	          msgListFile.write(filename+"\t"+"UNKNOW Error: "+e.getMessage()+"\n");
                      	          msgListFile.flush();
                      	         }
                      	        catch( Exception e1 ) 
                       	         {
                       	         e1.printStackTrace();
                       	          }
                 	         e.printStackTrace();
                 	       }
                         }//for
                        if (previousIDList!= null)
                        {
                        	
                        	Iterator<Integer> iter = previousIDList.iterator();
                        	int numOfDuplicate = 0;
                            while (iter.hasNext())
                            {
                           	Integer id = iter.next();
                           	if (currentIDList.contains(id)){
                           		numOfDuplicate++;
                           		System.out.println("id:"+id+"is duplicate!");
                           	}
                           	}
                            try{
                          	  if (numOfDuplicate == 20){
                          		msgListFile.write(filename+"\t"+"File Duplicate"+"\n");
                          		msgListFile.flush();
                                  }
                              else if (numOfDuplicate > 0){
                                	  msgListFile.write(filename+"\t"+numOfDuplicate+" Records Duplicate"+"\n");
                                	  msgListFile.flush();
                                  } 
                            }
                      	  catch( Exception e ) 
                    	      {
                    	         e.printStackTrace();
                    	       }
                        }//if
                        previousIDList = currentIDList;
                        currentIDList = null;
               
               System.gc() ;     
          	    }
              }
                  //finished one sub-dir
                  try
            	  {
            	  finishedDirFile.write(dir_children[index]+"\n");
            	  finishedDirFile.flush();
            	  System.out.println("finished sub-directory:"+dir_children[index]);
            	  }
            	  catch( Exception e ) 
          	      {
          	         e.printStackTrace();
          	       }
         	  
          }
    	  
        }
      try{
      msgListFile.close();}
      catch( Exception e ){
	         e.printStackTrace();}
      }
  
  }

  
  public static void main(String args[]) {
   
	if (args.length < 1)
	{
		System.out.println("Usage: getmsgID <Source file directoy>");
		System.exit(0) ;
	}
	try{  
	String rootDir = args[0];
	GetMsgIDApp insertapp = new GetMsgIDApp();
	
    insertapp.insertAllFile(rootDir);
    //insertapp.con.close();
	}
      catch( Exception e ) 
      {
         e.printStackTrace();
       }
  }
  
}
